

<!DOCTYPE html>
<html class="writer-html5" lang="zh" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>AutoThrottle 扩展 &mdash; Scrapy 2.3.0 文档</title>
  

  
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.custom.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster.bundle.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-shadow.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-punk.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-noir.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-light.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tooltipster-sideTip-borderless.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/micromodal.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/sphinx_rtd_theme.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/language_data.js"></script>
        <script src="../_static/js/hoverxref.js"></script>
        <script src="../_static/js/tooltipster.bundle.min.js"></script>
        <script src="../_static/js/micromodal.min.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="标杆管理" href="benchmarking.html" />
    <link rel="prev" title="部署蜘蛛" href="deploy.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html" class="icon icon-home" alt="Documentation Home"> Scrapy
          

          
          </a>

          
            
            
              <div class="version">
                2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">第一步</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intro/overview.html">Scrapy一目了然</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/install.html">安装指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/tutorial.html">Scrapy 教程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intro/examples.html">实例</a></li>
</ul>
<p class="caption"><span class="caption-text">基本概念</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="commands.html">命令行工具</a></li>
<li class="toctree-l1"><a class="reference internal" href="spiders.html">蜘蛛</a></li>
<li class="toctree-l1"><a class="reference internal" href="selectors.html">选择器</a></li>
<li class="toctree-l1"><a class="reference internal" href="items.html">项目</a></li>
<li class="toctree-l1"><a class="reference internal" href="loaders.html">项目加载器</a></li>
<li class="toctree-l1"><a class="reference internal" href="shell.html">Scrapy shell</a></li>
<li class="toctree-l1"><a class="reference internal" href="item-pipeline.html">项目管道</a></li>
<li class="toctree-l1"><a class="reference internal" href="feed-exports.html">Feed 导出</a></li>
<li class="toctree-l1"><a class="reference internal" href="request-response.html">请求和响应</a></li>
<li class="toctree-l1"><a class="reference internal" href="link-extractors.html">链接提取器</a></li>
<li class="toctree-l1"><a class="reference internal" href="settings.html">设置</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">例外情况</a></li>
</ul>
<p class="caption"><span class="caption-text">内置服务</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="logging.html">登录</a></li>
<li class="toctree-l1"><a class="reference internal" href="stats.html">统计数据集合</a></li>
<li class="toctree-l1"><a class="reference internal" href="email.html">发送电子邮件</a></li>
<li class="toctree-l1"><a class="reference internal" href="telnetconsole.html">远程登录控制台</a></li>
<li class="toctree-l1"><a class="reference internal" href="webservice.html">Web服务</a></li>
</ul>
<p class="caption"><span class="caption-text">解决具体问题</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../faq.html">常见问题</a></li>
<li class="toctree-l1"><a class="reference internal" href="debug.html">调试spiders</a></li>
<li class="toctree-l1"><a class="reference internal" href="contracts.html">蜘蛛合约</a></li>
<li class="toctree-l1"><a class="reference internal" href="practices.html">常用做法</a></li>
<li class="toctree-l1"><a class="reference internal" href="broad-crawls.html">宽爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="developer-tools.html">使用浏览器的开发人员工具进行抓取</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamic-content.html">选择动态加载的内容</a></li>
<li class="toctree-l1"><a class="reference internal" href="leaks.html">调试内存泄漏</a></li>
<li class="toctree-l1"><a class="reference internal" href="media-pipeline.html">下载和处理文件和图像</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">部署蜘蛛</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">AutoThrottle 扩展</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#design-goals">设计目标</a></li>
<li class="toctree-l2"><a class="reference internal" href="#how-it-works">它是如何工作的</a></li>
<li class="toctree-l2"><a class="reference internal" href="#throttling-algorithm">节流算法</a></li>
<li class="toctree-l2"><a class="reference internal" href="#settings">设置</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#autothrottle-enabled">AUTOTHROTTLE_ENABLED</a></li>
<li class="toctree-l3"><a class="reference internal" href="#autothrottle-start-delay">AUTOTHROTTLE_START_DELAY</a></li>
<li class="toctree-l3"><a class="reference internal" href="#autothrottle-max-delay">AUTOTHROTTLE_MAX_DELAY</a></li>
<li class="toctree-l3"><a class="reference internal" href="#autothrottle-target-concurrency">AUTOTHROTTLE_TARGET_CONCURRENCY</a></li>
<li class="toctree-l3"><a class="reference internal" href="#autothrottle-debug">AUTOTHROTTLE_DEBUG</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="benchmarking.html">标杆管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="jobs.html">作业：暂停和恢复爬行</a></li>
<li class="toctree-l1"><a class="reference internal" href="coroutines.html">协同程序</a></li>
<li class="toctree-l1"><a class="reference internal" href="asyncio.html">asyncio</a></li>
</ul>
<p class="caption"><span class="caption-text">扩展Scrapy</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="architecture.html">体系结构概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloader-middleware.html">下载器中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="spider-middleware.html">蜘蛛中间件</a></li>
<li class="toctree-l1"><a class="reference internal" href="extensions.html">扩展</a></li>
<li class="toctree-l1"><a class="reference internal" href="api.html">核心API</a></li>
<li class="toctree-l1"><a class="reference internal" href="signals.html">信号</a></li>
<li class="toctree-l1"><a class="reference internal" href="exporters.html">条目导出器</a></li>
</ul>
<p class="caption"><span class="caption-text">其余所有</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../news.html">发行说明</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributing.html">为 Scrapy 贡献</a></li>
<li class="toctree-l1"><a class="reference internal" href="../versioning.html">版本控制和API稳定性</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">Scrapy</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>AutoThrottle 扩展</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="autothrottle-extension">
<span id="topics-autothrottle"></span><h1>AutoThrottle 扩展<a class="headerlink" href="#autothrottle-extension" title="永久链接至标题">¶</a></h1>
<p>这是一个扩展，基于Scrapy服务器和您正在爬行的网站的负载，自动限制爬行速度。</p>
<div class="section" id="design-goals">
<h2>设计目标<a class="headerlink" href="#design-goals" title="永久链接至标题">¶</a></h2>
<ol class="arabic simple">
<li><p>对站点更好，而不是使用默认的下载延迟为零</p><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<ins class="adsbygoogle"
     style="display:block; text-align:center;"
     data-ad-layout="in-article"
     data-ad-format="fluid"
     data-ad-client="ca-pub-1466963416408457"
     data-ad-slot="8850786025"></ins>
<script>
     (adsbygoogle = window.adsbygoogle || []).push({});
</script></li>
<li><p>自动调整Scrapy到最佳的爬行速度，因此用户不必调整下载延迟来找到最佳的。用户只需要指定它允许的最大并发请求，其余的由扩展来完成。</p></li>
</ol>
</div>
<div class="section" id="how-it-works">
<span id="autothrottle-algorithm"></span><h2>它是如何工作的<a class="headerlink" href="#how-it-works" title="永久链接至标题">¶</a></h2>
<p>AutoThrottle 扩展动态调整下载延迟，使蜘蛛发送 <a class="reference internal" href="#std-setting-AUTOTHROTTLE_TARGET_CONCURRENCY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code></a> 平均每个远程网站的并发请求。</p>
<p>它使用下载延迟来计算延迟。主要思想如下：如果服务器需要 <code class="docutils literal notranslate"><span class="pre">latency</span></code> 响应时间为秒，客户端应每秒钟发送一个请求 <code class="docutils literal notranslate"><span class="pre">latency/N</span></code> 秒有 <code class="docutils literal notranslate"><span class="pre">N</span></code> 并行处理的请求。</p>
<p>不需要调整延迟，只需设置一个小的固定下载延迟，并对使用 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_DOMAIN"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_DOMAIN</span></code></a> 或 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_IP"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_IP</span></code></a> 选项。它会产生类似的效果，但有一些重要的区别：</p>
<ul class="simple">
<li><p>因为下载延迟很小，偶尔会有突发的请求；</p></li>
<li><p>通常，非200（错误）响应的返回速度比常规响应快，因此，只要有一个较小的下载延迟和硬并发限制，当服务器开始返回错误时，爬虫程序将更快地向服务器发送请求。但这与爬虫应该做的相反——如果出现错误，放慢速度更有意义：这些错误可能是由高请求率引起的。</p></li>
</ul>
<p>AutoThrottle 没有这些问题。</p>
</div>
<div class="section" id="throttling-algorithm">
<h2>节流算法<a class="headerlink" href="#throttling-algorithm" title="永久链接至标题">¶</a></h2>
<p>AutoThrottle 算法根据以下规则调整下载延迟：</p>
<ol class="arabic simple">
<li><p>蜘蛛总是以下载延迟开始 <a class="reference internal" href="#std-setting-AUTOTHROTTLE_START_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_START_DELAY</span></code></a> ；</p></li>
<li><p>当收到响应时，目标下载延迟计算为 <code class="docutils literal notranslate"><span class="pre">latency</span> <span class="pre">/</span> <span class="pre">N</span></code> 在哪里？ <code class="docutils literal notranslate"><span class="pre">latency</span></code> 是响应的延迟，并且 <code class="docutils literal notranslate"><span class="pre">N</span></code> 是 <a class="reference internal" href="#std-setting-AUTOTHROTTLE_TARGET_CONCURRENCY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code></a> .</p></li>
<li><p>下一个请求的下载延迟设置为上一个下载延迟和目标下载延迟的平均值；</p></li>
<li><p>不允许非200响应的延迟减少延迟；</p></li>
<li><p>下载延迟不能小于 <a class="reference internal" href="settings.html#std-setting-DOWNLOAD_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DOWNLOAD_DELAY</span></code></a> 或大于 <a class="reference internal" href="#std-setting-AUTOTHROTTLE_MAX_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_MAX_DELAY</span></code></a></p></li>
</ol>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p>autothrottle扩展支持并发和延迟的标准  Scrapy   设置。这意味着它将尊重 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_DOMAIN"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_DOMAIN</span></code></a> 和 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_IP"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_IP</span></code></a> 选项，并且从不将下载延迟设置为低于 <a class="reference internal" href="settings.html#std-setting-DOWNLOAD_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DOWNLOAD_DELAY</span></code></a> .</p>
</div>
<p id="download-latency">在scrappy中，下载延迟是以建立TCP连接和接收HTTP头之间所经过的时间来度量的。</p>
<p>注意，在一个合作的多任务环境中，这些延迟很难精确测量，因为scrapy可能正忙于处理spider回调，例如，无法参加下载。然而，这些延迟仍然应该对Scrapy（最终是服务器）有多忙给出一个合理的估计，并且这个扩展是在这个前提下构建的。</p>
</div>
<div class="section" id="settings">
<h2>设置<a class="headerlink" href="#settings" title="永久链接至标题">¶</a></h2>
<p>用于控制 AutoThrottle 扩展的设置为：</p>
<ul class="simple">
<li><p><a class="reference internal" href="#std-setting-AUTOTHROTTLE_ENABLED"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_ENABLED</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-AUTOTHROTTLE_START_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_START_DELAY</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-AUTOTHROTTLE_MAX_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_MAX_DELAY</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-AUTOTHROTTLE_TARGET_CONCURRENCY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code></a></p></li>
<li><p><a class="reference internal" href="#std-setting-AUTOTHROTTLE_DEBUG"><code class="xref std std-setting docutils literal notranslate"><span class="pre">AUTOTHROTTLE_DEBUG</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_DOMAIN"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_DOMAIN</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_IP"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_IP</span></code></a></p></li>
<li><p><a class="reference internal" href="settings.html#std-setting-DOWNLOAD_DELAY"><code class="xref std std-setting docutils literal notranslate"><span class="pre">DOWNLOAD_DELAY</span></code></a></p></li>
</ul>
<p>有关详细信息，请参阅 <a class="reference internal" href="#autothrottle-algorithm"><span class="std std-ref">它是如何工作的</span></a> .</p>
<div class="section" id="autothrottle-enabled">
<span id="std-setting-AUTOTHROTTLE_ENABLED"></span><span id="std:setting-AUTOTHROTTLE_ENABLED"></span><h3>AUTOTHROTTLE_ENABLED<a class="headerlink" href="#autothrottle-enabled" title="永久链接至标题">¶</a></h3>
<p>违约： <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
<p>启用AutoThrottle 扩展。</p>
</div>
<div class="section" id="autothrottle-start-delay">
<span id="std-setting-AUTOTHROTTLE_START_DELAY"></span><span id="std:setting-AUTOTHROTTLE_START_DELAY"></span><h3>AUTOTHROTTLE_START_DELAY<a class="headerlink" href="#autothrottle-start-delay" title="永久链接至标题">¶</a></h3>
<p>违约： <code class="docutils literal notranslate"><span class="pre">5.0</span></code></p>
<p>初始下载延迟（秒）。</p>
</div>
<div class="section" id="autothrottle-max-delay">
<span id="std-setting-AUTOTHROTTLE_MAX_DELAY"></span><span id="std:setting-AUTOTHROTTLE_MAX_DELAY"></span><h3>AUTOTHROTTLE_MAX_DELAY<a class="headerlink" href="#autothrottle-max-delay" title="永久链接至标题">¶</a></h3>
<p>违约： <code class="docutils literal notranslate"><span class="pre">60.0</span></code></p>
<p>在高延迟情况下设置的最大下载延迟（秒）。</p>
</div>
<div class="section" id="autothrottle-target-concurrency">
<span id="std-setting-AUTOTHROTTLE_TARGET_CONCURRENCY"></span><span id="std:setting-AUTOTHROTTLE_TARGET_CONCURRENCY"></span><h3>AUTOTHROTTLE_TARGET_CONCURRENCY<a class="headerlink" href="#autothrottle-target-concurrency" title="永久链接至标题">¶</a></h3>
<p>违约： <code class="docutils literal notranslate"><span class="pre">1.0</span></code></p>
<p>Scrapy的平均请求数应与远程网站并行发送。</p>
<p>默认情况下，autothrottle会调整延迟以向每个远程网站发送单个并发请求。将此选项设置为更高的值（例如 <code class="docutils literal notranslate"><span class="pre">2.0</span></code> ）以增加远程服务器的吞吐量和负载。下层 <code class="docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code> 价值（例如） <code class="docutils literal notranslate"><span class="pre">0.5</span></code> ）让爬虫人更加保守和礼貌。</p>
<p>注意 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_DOMAIN"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_DOMAIN</span></code></a> 和 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_IP"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_IP</span></code></a> 启用 AutoThrottle 扩展功能时，仍会遵循选项。这意味着如果 <code class="docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code> 设置为大于的值 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_DOMAIN"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_DOMAIN</span></code></a> 或 <a class="reference internal" href="settings.html#std-setting-CONCURRENT_REQUESTS_PER_IP"><code class="xref std std-setting docutils literal notranslate"><span class="pre">CONCURRENT_REQUESTS_PER_IP</span></code></a> ，爬虫程序将无法达到此数量的并发请求。</p>
<p>在每个给定的时间点上，scrapy可以发送的并发请求多于或少于 <code class="docutils literal notranslate"><span class="pre">AUTOTHROTTLE_TARGET_CONCURRENCY</span></code> ；这是爬虫尝试接近的建议值，而不是硬限制。</p>
</div>
<div class="section" id="autothrottle-debug">
<span id="std-setting-AUTOTHROTTLE_DEBUG"></span><span id="std:setting-AUTOTHROTTLE_DEBUG"></span><h3>AUTOTHROTTLE_DEBUG<a class="headerlink" href="#autothrottle-debug" title="永久链接至标题">¶</a></h3>
<p>违约： <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
<p>启用 AutoThrottle 调试模式，该模式将显示收到的每个响应的统计信息，以便您可以看到如何实时调整节流参数。</p>
</div>
</div>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="benchmarking.html" class="btn btn-neutral float-right" title="标杆管理" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="deploy.html" class="btn btn-neutral float-left" title="部署蜘蛛" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; 版权所有 2008–2020, Scrapy developers
      <span class="lastupdated">
        最后更新于 10月 18, 2020.
      </span>

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
  
 
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();

analytics.ready(function () {
    ga('require', 'linker');
    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
});
</script>


</body>
</html>